# coding:utf-8

import scrapy
from bs4 import BeautifulSoup
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import Rule, CrawlSpider

from scrapy_redis.spiders import RedisCrawlSpider
from itjuzi.items import CompanyItem


#class ITjuziSpider(RedisCrawlSpider):
class ITjuziSpider(RedisCrawlSpider):
    name = 'itjuzi'
    allowed_domains = ['www.itjuzi.com']
    #start_urls = ['http://www.itjuzi.com/company?page=1/']
    redis_key = 'itjuzispider:start_urls'

    headers = {
        "Host" : "www.itjuzi.com",
        "Connection" : "keep-alive",
        "Upgrade-Insecure-Requests" : "1",
        "User-Agent" : "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36",
        "Accept" : "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
        "Accept-Encoding" : "gzip, deflate, sdch",
        "Accept-Language" : "zh-CN,zh;q=0.8,en;q=0.6",
        # "Referer" : "http://www.itjuzi.com/company",
        "Cookie" : "acw_tc=AQAAAFWStB1IzQsAbrTqenvRNG/VyFHl; gr_user_id=9acdce80-6392-4d8f-a68d-87fa36a579d0; gr_session_id_eee5a46c52000d401f969f4535bdaa78=f0300833-22fe-4c29-9227-a13724ba673d; Hm_lvt_1c587ad486cdb6b962e94fc2002edf89=1510713361; Hm_lpvt_1c587ad486cdb6b962e94fc2002edf89=1510713361; _ga=GA1.2.253936966.1510713361; _gid=GA1.2.171048596.1510713361; _gat=1",
        "If-Modified-Since" : "Wed, 15 Nov 2017 02:37:47 GMT"
    }

    rules = [
        # 获取每一页的链接
        Rule(link_extractor=LinkExtractor(allow=('/company\?page=\d+'))),
        Rule(link_extractor=LinkExtractor(allow=('/company/foreign\?page=\d+'))),
        # 获取每一个公司的详情
        Rule(link_extractor=LinkExtractor(allow=('/company/\d+')), callback='parse_item')
    ]


    def make_requests_from_url(self, url):
        return scrapy.Request(url,
                headers = self.headers,
                dont_filter = True)

    def parse_item(self, response):
        print ("======================" + response.url)
        item = CompanyItem()
        #取出后面的数字编号
        item['info_id'] = response.url.split('/')[-1:][0]
        item['company_name'] = self.get_company_name(response).replace('\t','')
        item['slogan'] = self.get_slogan(response)

        item['company_weibo'] = self.get_company_weibo(response)
        item['home_page'] = self.get_home_page(response)
        item['tags'] = self.get_tags(response)

        item['company_intro'] = self.get_company_intro(response)
        item['company_full_name'] = self.get_company_full_name(response)
        item['found_time'] = self.get_found_time(response)
        item['company_size'] = self.get_company_size(response)
        item['company_status'] = self.get_company_status(response)

        # item['tz_info'] = tz_list
        # item['tm_info'] = tm_list
        # item['pdt_info'] = pdt_list
        yield item

    def get_company_name(self,response):
        res = response.xpath("//div[@class='rowhead feedback-btn-parent']//h1[@class='seo-important-title']/text()").extract()
        company_name = ""
        if res:
            company_name = res[0].strip()
        else:
            company_name = "NULL"
        return company_name

    def get_slogan(self,response):
        res = response.xpath("//div[@class='rowhead feedback-btn-parent']//h2[@class='seo-slogan']/text()").extract()
        slogan = ""
        if res:
            slogan = res[0].strip()
        else:
            slogan = "NULL"
        return slogan

    def get_company_weibo(self,response):
        res = response.xpath("//div[@class='link-line']/a[1]/@href").extract()
        company_weibo = ""
        if res:
            company_weibo = res[0]
        else:
            company_weibo = "NULL"
        return company_weibo

    def get_home_page(self,response):
        res = response.xpath("//div[@class='link-line']/a[@target='_blank']/@href").extract()
        home_page = ""
        if res:
            if res[0].find('weibo') == -1:
                home_page = res[0]
            else:
                res = response.xpath("//div[@class='link-line']/a[2]/@href").extract()
                if res:
                    home_page = res[0]
                else:
                    res = "NULL"
        else:
            home_page = "NULL"
        return home_page


    def get_tags(self,response):
        res = response.xpath('//div[@class="tagset dbi c-gray-aset tag-list"]')
        tags = ""
        if res:
            for each in res:
               tags = tags + each.xpath("./a/text()").extract()[0] + ","
        else:
            tags = "NULL"
        return tags


    def get_company_intro(self,response):
        res = response.xpath("//div[@class='block-inc-info on-edit-hide']/div[2]/div[2]/text()").extract()
        company_intro = ""
        if res:
            company_intro = res[0].strip()
        else:
            company_intro = "NULL"
        return company_intro

    def get_company_full_name(self,response):
        res = response.xpath("//div[@class='block']/div[@class='des-more']/h2/text()").extract()
        company_full_name = ""
        if res:
            company_full_name = res[0].strip()
        else:
            company_full_name = "NULL"
        return company_full_name

    def get_found_time(self, response):
        res = response.xpath("//div[@class='block-inc-info on-edit-hide']/div[3]/div[1]/h3[1]/span/text()").extract()
        found_time = ""
        if res:
            found_time = res[0]
        else:
            found_time = "NULL"
        return found_time

    def get_company_size(self, response):
        res = response.xpath("//div[@class='block-inc-info on-edit-hide']/div[3]/div[1]/h3[2]/span/text()").extract()
        company_size = ""
        if res:
            company_size = res[0].strip()
        else:
            company_size = "NULL"
        return company_size

    def get_company_status(self, response):
        res = response.xpath("//div[@class='block-inc-info on-edit-hide']/div[3]/div[1]/span/text()").extract()
        company_status = ""
        if res:
            company_status = res[0].strip()
        else:
            company_status = "NULL"
        return company_status

    # def get_tz_info(self, response):
    #     res = response.xpath("//tr[@class='feedback-btn-parent']").extract()
    #     tz_info = []
    #     if res:
    #         for each in res:
    #             pass
    #     else:
    #         tz_info = "NULL"
    #     return
